# -*- coding:utf8 -*-
from db_manager import DBSession
from db_manager import MongoDB
import settings
from utils import help
import json
import datetime
from datetime import date
import re


"""
json转化中datetime转化方法
"""


class DateEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, datetime.datetime):
            return obj.strftime('%Y-%m-%d %H:%M:%S')
        elif isinstance(obj, date):
            return obj.strftime("%Y-%m-%d")
        else:
            return json.JSONEncoder.default(self, obj)


"""
词频清洗和统计类
"""


class FilterWord(object):
    def __init__(self):
        self.session = DBSession(settings.db_config)
        self.mongodb_session = MongoDB(settings.mongo_db_config)
        self.path = './filter_words.txt'
        self.filter_words = help.get_all_filter_words(self.path)
        self.counts = self.get_info_word_counts()

    def get_info_word_counts(self):
        """
        得到info_word表总行数...
        :return:
        """
        counts = self.session.execute_query(settings.query_info_word_counts)[0]["COUNT(*)"]
        return counts

    def get_info_word_msg(self, from_num=0, batch_size=10):
        # 每10条一批插入
        result = self.session.execute_query(settings.query_all_info_word_by_page % (from_num, batch_size))
        return result

    def is_filter_word(self, word):
        """
        判断某个单词是否是停词表中的单词
        :param word:
        :return:
        """
        if word in self.filter_words:
            return True
        return False

    def cal_word_freq(self, dic_words, word):
        """
        统计词频
        :param dic_words: 记录的字典
        :param word: 需要加入字典的内容
        :return: 加入word后的新字典
        """
        if word in dic_words:
            dic_words[word] = dic_words[word] + 1
        else:
            dic_words[word] = 1

        return dic_words

    def is_integer(self, word):
        """
        判断是否是整数，整数就过滤掉
        :param word:
        :return:
        """
        pat_integer = '\d+'
        if re.match(pat_integer, word):
            return True
        return False

    def clear_words(self, words):
        """
        将数据清洗一遍并统计词频
        :param words:
        :return: 返回一个字典，内容是每个词的频率
        """
        result = dict()
        words = words.split('/')
        for word in words:
            word = word.lower()
            if word is ' ' or len(word) < 2 or '.' in word:
                # 这就可能是符号或者空格，就不要了
                continue
            elif self.is_filter_word(word):
                # 如果是需要过滤的单词，也不要了
                continue
            elif self.is_integer(word):
                # 如果是整数也不要了
                continue
            else:
                # 说明是有用的单词，统计词频
                result = self.cal_word_freq(result, word)
        return result

    def change_info_word_2_freq_dict(self, info_word):
        """
        将info_word中的每一条记录变成统计词频后的dict
        :param info_word: 数据库中的每一条记录
        :return:  返回一个dict
        """
        pub_time = info_word["pub_time"]
        words = info_word["cut_content"]
        result_dict = self.clear_words(words)
        if result_dict is not None:
            result_dict["pub_time"] = pub_time
        else:
            return None
        return result_dict

    def get_union_data_by_date(self, from_date=None, to_date=None, date_nums=40):
        """
        按照时间段聚合数据
        :param from_date: 开始日期,默认为空
        :param to_date: 结束日期,默认为空
        :param date_nums: 默认天数,默认为40
        :return: result,为该时间段的所有的单词集合
        """
        result = dict()
        # 得到这个时间段的数据,类型list
        data = self.mongodb_session.query_all_data_by_date(from_date=from_date, to_date=to_date, date_nums=date_nums)
        for item in data:
            for k, v in item.items():
                if '_id' in k:
                    continue
                if 'pub_time' in k:
                    if from_date is None:
                        from_date = help.get_before_date_by_delta(current_date=to_date, delta_days=date_nums)
                    result["pub_time"] = from_date + " -> " + to_date
                    continue
                if k in result:
                    result[k] = result[k] + v
                else:
                    result[k] = v
        return result

    def get_union_labels(self, union_dict):
        """
        根据合并后的数据字典提取里面的keys,主要作为每一列的标题
        :param union_dict: 合并数据后的字典
        :return: 返回的是列的集合
        """
        result = []
        for k, v in union_dict.items():
            if 'pub_time' not in k:
                result.append(k)
        return result

    def get_union_arr_by_date(self, from_date=None, to_date=None, date_nums=40, labels=[]):
        """
        通过时间段的数据聚合后得到矩阵的某一行如: pub_time::1::2::3::0
        :param from_date: 开始日期,默认为空
        :param to_date: 结束日期,默认为空
        :param date_nums: 默认天数,默认为40
        :param labels: 对应的列名
        :return: pub_time::1::2::3::0
        """
        result = ''
        init_arr = []
        # 初始化result矩阵，空间复杂度O(n)
        init_arr = help.init_by_other_arr(init_arr, labels)
        data = self.get_union_data_by_date(from_date=from_date, to_date=to_date, date_nums=date_nums)
        for k, v in data.items():
            index = help.find_index_in_arr(word=k, arr=labels)
            if 'pub_time' in k:
                continue
            if index is not -1:
                init_arr[index] = v
            else:
                print "ERROR: can not find %s in labels %s" % (k, ','.join(labels))
        result = from_date + " -> " + to_date + "::" + str(init_arr).replace('[', '').replace(']', '').replace(',', '::').\
            replace(' ', '').replace('\n', '') + '\n'
        # 纯矩阵
        pure_arr = str(init_arr).replace('[', '').replace(']', '').replace(',', '::').\
            replace(' ', '').replace('\n', '') + '\n'
        return result, pure_arr

    def union_data_run_and_save(self, from_date=None, to_date=None, date_nums=40, save_path='./data/', date_list=None):
        """
        运行并保存txt文件
        :param from_date: 总日期的起始时间
        :param to_date: 总日期的结束时间
        :param date_nums: 日期天数
        :param date_list: 中间矩阵每条数据对应的时间间隔
        :param save_path: 存储的目录
        :return:
        """
        if from_date is None:
            from_date = help.get_before_date_by_delta(to_date, date_nums)
        union_labels_file_name = from_date.split(' ')[0] + "_" + to_date.split(' ')[0] + " labels.txt"
        union_arr_file_name = from_date.split(' ')[0] + "_" + to_date.split(' ')[0] + " arr.txt"
        union_pure_arr_file_name = from_date.split(' ')[0] + "_" + to_date.split(' ')[0] + " pure_arr.txt"
        # 存储列名称的路径
        labels_path = save_path + union_labels_file_name

        # 存储矩阵文件的路径
        arr_path = save_path + union_arr_file_name

        # 存储纯矩阵文件路径
        pure_arr_path = save_path + union_pure_arr_file_name
        # 得到40天的数据集
        result = self.get_union_data_by_date(from_date=from_date, to_date=to_date, date_nums=date_nums)
        union_labels = self.get_union_labels(result)
        union_labels_write_msg = '::'.join(union_labels).replace('\n', '')
        with open(labels_path, 'wb') as fs:
            fs.write(union_labels_write_msg)

        # 矩阵文件存储
        for date_msg in date_list:
            f, t = date_msg.split('_')
            union_data, pure_arr = self.get_union_arr_by_date(from_date=f, to_date=t, date_nums=date_nums, labels=union_labels)
            with open(arr_path, 'ab') as fs:
                fs.write(union_data)
            with open(pure_arr_path, 'ab') as fs:
                fs.write(pure_arr)

    def save_dict_2_json(self, dict_msg, path='./data/result.json'):
        with open(path, 'wb') as fs:
            dict_msg = json.dumps(dict_msg, cls=DateEncoder)
            fs.write(dict_msg)

    def save_to_mongodb(self, dict_msg):
        """
        将数据存储到mongodb数据库中
        :param dict_msg: 需要存储的数据
        :return:
        """
        result = self.mongodb_session.insert(dict_msg)
        if result > 0:
            pass
            # print "Insert Success..."
        else:
            print "Insert Failed..."


if __name__ == "__main__":
    f = FilterWord()
    from_date = '2013-01-31 00:00:00'
    to_date = '2013-02-03 00:00:00'

    date_list = [
        '2013-01-31 01:00:00_2013-01-31 03:00:00',
        '2013-01-31 03:00:00_2013-01-31 05:00:00',
        '2013-01-31 05:00:00_2013-01-31 10:00:00',
        '2013-01-31 10:00:00_2013-01-31 20:00:00',
        '2013-01-31 21:00:00_2013-02-01 00:00:00',
        '2013-02-01 00:00:00_2013-02-01 10:00:00',
        '2013-02-01 10:00:00_2013-02-02 00:00:00'
    ]
    f.union_data_run_and_save(to_date=to_date, date_list=date_list)


